{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch as t\n",
    "from torch.autograd import Variable as V\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1.,  1.,  1.,  1.],\n",
       "         [ 1.,  1.,  1.,  1.],\n",
       "         [ 1.,  1.,  1.,  1.]]), True, True)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# requires_grad's default is false.If a leaf is True,its depended nodes are all True.\n",
    "a = V(t.ones(3,4), requires_grad = True)\n",
    "b = V(t.ones(3,4))\n",
    "c = a+b\n",
    "d = c.sum()  # calculate the sum\n",
    "d.backward()  #back-propagation\n",
    "a.grad,c.requires_grad,c.grad is None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, True, False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.is_leaf,b.is_leaf,c.is_leaf  # autograd only calculate the leaf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(24.), tensor(24.))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.data.sum() , c.sum()  # first is tensor,second is variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.5345,  0.0538,  1.2794],\n",
      "        [ 0.0284,  0.0145,  0.9274]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1.4622,  0.4641, -2.2622],\n",
       "         [-0.3371, -0.2405,  1.9260]]), tensor([[ 1.4622,  0.4641, -2.2622],\n",
       "         [-0.3371, -0.2405,  1.9260]]))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's do an example\n",
    "def f(x):\n",
    "    y = x**2\n",
    "    return y\n",
    "def gradf(x):\n",
    "    dx = 2*x\n",
    "    return dx\n",
    "x = V(t.randn(2,3),requires_grad = True)\n",
    "y = f(x)\n",
    "print(y)\n",
    "y.backward(t.ones(x.size()))\n",
    "x.grad,gradf(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = V(t.ones(1))\n",
    "b = V(t.rand(1),requires_grad=True)\n",
    "w = V(t.rand(1),requires_grad=True)\n",
    "y = w*x\n",
    "z = y+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, True, <AddBackward1 at 0x2552a80ec88>, <MulBackward1 at 0x255299c5b38>)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#grad_fn is used to show the backward functions.grad_fn.next_functions is used to save input of grad_fn.\n",
    "y.requires_grad,z.requires_grad,z.grad_fn,y.grad_fn  #We can see z and y's backward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((<MulBackward1 at 0x255299c5b38>, 0), (<AccumulateGrad at 0x255299c5518>, 0))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.grad_fn.next_functions  # we can see z[0][0] = y.grad_fn,z[1][0] = b.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((<AccumulateGrad at 0x255299c50f0>, 0), (None, 0))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.grad_fn.next_functions  #y[0][0]=w.grad  y[1][0]=x.grad(none)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(None, None)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# leaf's grad_fn is none\n",
    "w.grad_fn,x.grad_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# retain_graph can stop to deleting the buffer\n",
    "#multi-backward will make the grad accmulated.\n",
    "z.backward(retain_graph = True)\n",
    "w.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pytoch uses Dynamic graphics.Every forward will recreate calculate picture.So we can use python's control setence(if,while,for) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 1.,  1.,  1.]),)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = V(t.ones(3),requires_grad = True)\n",
    "w = V(t.ones(3),requires_grad = True)\n",
    "y = x*w\n",
    "z = y.sum()\n",
    "# because after backward,y's grad will automaticly delete.In order to find the y'grad,we can use autograd.grad() or hook\n",
    "t.autograd.grad(z,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y's grad:\n",
      " tensor([ 1.,  1.,  1.])\n"
     ]
    }
   ],
   "source": [
    "def hook1(grad):\n",
    "    print(\"y's grad:\\n\",grad)\n",
    "x = V(t.ones(3),requires_grad = True)\n",
    "w = V(t.ones(3),requires_grad = True)\n",
    "y = x*w\n",
    "hook_handle = y.register_hook(hook1)\n",
    "z = y.sum()\n",
    "z.backward()\n",
    "hook_handle.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.,  4.,  6.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's see the grad_variables. z.backward() = y.backward(grad_y)\n",
    "x = V(t.arange(0,3),requires_grad = True)\n",
    "y = x**2 + x*2\n",
    "z = y.sum()\n",
    "z.backward()\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.,  4.,  6.])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = V(t.arange(0,3),requires_grad = True)\n",
    "y = x**2 + x*2\n",
    "z = y.sum()\n",
    "grad_y = V(t.ones(1))\n",
    "y.backward(grad_y)\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
